package com.itheima.jdspider;

import com.itheima.jdspider.pojo.Item;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.xml.crypto.Data;
import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.http.HttpClient;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;

public class JDSpider {
    public static void main(String[] args) throws IOException, SQLException {
        /*
        https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page=1&s=1&click=0
        https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page=3&s=56&click=0
        https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page=5&s=117&click=0
        https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page=7&click=0
        https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page=9&s=236&click=0
        https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page=13&s=357&click=0
         */


        int page=1;
        int count=0;
        CloseableHttpClient client=HttpClients.createDefault();
        while (page<180) {


            String indexUrl = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page="+page+"&click=0";

            HttpGet httpGet = new HttpGet(indexUrl);

            httpGet.setHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36");

            CloseableHttpResponse resp = client.execute(httpGet);

            HttpEntity entity = resp.getEntity();
            String html = EntityUtils.toString(entity);


            resp.close();

            //解析数据
            Document document = Jsoup.parse(html);

            //获取所有商品的li标签
            Elements lis = document.select("#J_goodsList>[class=gl-warp clearfix]>li");
            ArrayList<Item> items = new ArrayList<>();
            for (Element li : lis) {
                //获取每个li的图片

                Elements imgs = li.select(".gl-i-wrap>.p-img>a>img");
                String imgUrl = "https:" + imgs.attr("data-lazy-img");

                //根据图片地址发送请求获取字节流
                HttpGet imgGet = new HttpGet(imgUrl);
                CloseableHttpResponse imgResp = client.execute(imgGet);


                HttpEntity entity1 = imgResp.getEntity();
                InputStream imgInput = entity1.getContent();
                String imgFileName = "C:\\Users\\86186\\Desktop\\jd\\" + UUID.randomUUID().toString() + imgUrl.substring(imgUrl.lastIndexOf("."));
                FileOutputStream outputStream = new FileOutputStream(imgFileName);
                byte[] bytes = new byte[1024];
                int len;
                while ((len = imgInput.read(bytes)) != -1) {
                    outputStream.write(bytes, 0, len);
                }

                outputStream.close();
                imgInput.close();
                imgResp.close();

                String sku = li.attr("data-sku");
                String spu = li.attr("data-spu");
                if (spu == null || "".equals(spu)) {
                    spu = sku;
                }

                //商品名称
                Elements name = li.select(".gl-i-wrap>div[class=p-name p-name-type-2]>a>em");
                String title = name.text();
                //商品价格

                Elements eleprice = li.select(".gl-i-wrap>.p-price>strong>i");
                String price = eleprice.text();


                //商品的url
                Elements p_url = li.select(".gl-i-wrap>.p-img>a");
                String url = "https:" + p_url.attr("href");

                //封装数据
                SimpleDateFormat daf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");

                Item item = new Item(null,
                        Long.parseLong(spu),
                        Long.parseLong(sku),
                        title, Double.parseDouble(price),
                        imgFileName, url,
                        daf.format(new Date()),
                        daf.format(new Date()));
                //封装到集合中
                items.add(item);

            }
            count=count+ items.size();


            JDItemDao jdItemDao = new JDItemDao();
            jdItemDao.saveItem(items);

            page+=2;
        }
        System.out.println("获取到" +count );

        client.close();

    }
}
